package pais.robot;

import java.io.IOException;
import java.io.Reader;
import java.net.MalformedURLException;

import pais.db.HtmlPage;
import pais.db.HtmlPageTable;
import pais.db.Keyword;
import pais.db.KeywordTable;
import pais.db.PAISDB;
import pais.db.PAISSystemTable;
import pais.robot.http.HttpConnection;
import pais.robot.keyextr.KeywordExtractor;
import pais.robot.keyextr.KeywordListener;
import pais.robot.keyextr.SimpleKeywordExtractor;
import pais.robot.parser.HtmlParser;
import pais.robot.parser.ParseException;
import pais.robot.parser.TextListener;
import pais.robot.parser.TitleListener;
import pais.robot.parser.TokenMgrError;
import pais.robot.parser.URLListener;
import papos.rowset.AdvRandomAccessFile;
import papos.rowset.KeyNameNotFoundException;
import papos.rowset.StorageSystem;
import pasys.Pasys;
import pasys.error.PaposException;
import sun.misc.Signal;
import sun.misc.SignalHandler;


class Option {
  String dbPath_  ;        // The DB Path where indexed files are created
  String startURL_;        // The URL to begin to index
  long   executionDurationMS_; // How long the Robot continue to index pages?
}

public class DocRobot {
  public static int BUFFER_PAGE_COUNT = 100000;
  public static Option OPT = new Option();
  /* When DocRobot should exit immediately, this flag is turned on(true) */
  private boolean exit_ = false; 
  private PAISDB db_ = null;
  public void init() {
    try {
      System.err.println("DocRobot is using charset <" + Pasys.PAIS_CHAR_SET.name()+">" );
      System.setProperty("file.encoding", "UTF-8");
  
      db_ = PAISDB.getInstance(OPT.dbPath_);
      // Open database with read-write mode
      db_.open(true, BUFFER_PAGE_COUNT);
      System.out.println("Opened PAIS DB Successfully.");
      new ExitSignalHandler("ABRT");
      new ExitSignalHandler("INT");
      new ExitSignalHandler("TERM");
      System.out.println("Signal handlers registered successfully.");
    } catch (PaposException e) {
      e.printStackTrace();
      System.exit(-1);
    }
  }
  public synchronized void fini() {
    if ( db_ != null) {
      try {
        db_.close();
        db_ = null;
        System.err.println("Closed PAIS DB Successfully.");
      } catch (PaposException e) {
        e.printStackTrace();
        System.exit(-1);
      }
    }
  }
  public boolean isDBopen() {
    return db_ != null;
  }

  public class ExitSignalHandler implements SignalHandler {
    private String signalName_;
    private boolean isHandlingSignal_ = false;
    public ExitSignalHandler(String signalName) {
      signalName_ = signalName;
      install( signalName_ );
    }
    public void install(String signalName) {
      Signal signal = new Signal(signalName);
      Signal.handle(signal, this);
    }
    public void handle(Signal sig) {
      try {
        if (isHandlingSignal_) {
          // The signal is already being handled.
          return;
        }
        isHandlingSignal_ = true;
        System.err.println("Handling Signal("+signalName_+")");
        exit_ = true;
        while( isDBopen() ) {
          try {
            Thread.sleep(100);
          } catch (InterruptedException e){
          }
        }
        System.err.println("Exit Signal Handled Successfully.");
      } catch (Exception e) {
        System.err.println("Exit Signal Handler Failed, reason " + e);
      }
    }
  };

  public static void main(String args[]) {
    DocRobot robot = new DocRobot();
    if ( parseArgs(args) ) {
      robot.init();
      try {
        // Step 1. Index the page with given URL
        robot.indexHtmlPage(OPT.startURL_);
        // 	Step 2. Index all pages found from <A HREF=..> link, but not indexed yet.
        robot.indexUnIndexedPages();
      } catch (PaposException e) {
        e.printStackTrace();
      } catch (OutOfMemoryError e) {
        System.err.println("Out Of Memory Error");
        Pasys.printMemStat();
        e.printStackTrace();
      } finally {
        robot.fini();
      }
    } else {
      printUsage();
    }
  }
  /**
   * Index HTML Pages that were found from links in indexed pages, 
   * but not indexed yet.
   */
  public void indexUnIndexedPages() throws PaposException {
    try {
      long startTimeMS = System.currentTimeMillis();
      // The first page is already indexed
      long indexedPageCount = 0;
      long trialCount = 0;
      StorageSystem storage = HtmlPageTable.getInstance().getStorage();
      HtmlPage page = new HtmlPage();
      // Search all Pages that have its changed date not set yet.

      page.setChangeDate(HtmlPage.NULL_CHANGE_DATE);
      if ( storage.find("changeDate_", page) ) {
        while( true ) {
          System.out.print( "[" );
          System.out.print( trialCount );
          System.out.print( "]" );
          System.out.println(" Indexing ========> "+page.getURL());
          Pasys.printMemStat();

          trialCount ++;
          if( indexHtmlPage( page ) )
            indexedPageCount ++;

          if ( exit_ ) {
            System.out.println("Immediate Exit Detected. Stopping Job");
            break;
          }
          if ( isTimeOver( startTimeMS ) ) {
            break;
          }
          if ( (trialCount % 100) == 0) {
            PAISDB.getInstance(null).sync();
            printStat(startTimeMS, trialCount, indexedPageCount);
          }
          page.setChangeDate(HtmlPage.NULL_CHANGE_DATE);
          if ( ! storage.find("changeDate_", page) ) {
            break;
          }
        }
      }
      PAISDB.getInstance(null).sync();
      printStat(startTimeMS, trialCount, indexedPageCount);
    } catch (KeyNameNotFoundException e) {
      System.err.println("Invalid Key Field Name for HtmlPageTable : changeDate_");
      e.printStackTrace();
      System.exit(-1);
    }
  }
  private void printStat(long startTimeMS, long trialCount, long indexedPageCount) {
    long durationSeconds = executedDurationMS( startTimeMS) / 1000;
    double processPercent = (double)indexedPageCount * 100 / (double)trialCount; 
    System.out.println("Indexed "+indexedPageCount+" Pages among "+trialCount+" Pages ( "+processPercent+"% ).");
    System.out.println("DocRobot executed for "+ durationSeconds +" Seconds");
    System.out.println("DocRobot speed : "+(double)indexedPageCount/(double)durationSeconds+"(Pages/Second)");
  }
  private long executedDurationMS(long startTimeMS) {
    return System.currentTimeMillis() - startTimeMS;
  }
  private boolean isTimeOver(long startTimeMS ) {
    return executedDurationMS(startTimeMS) >= OPT.executionDurationMS_ ;
  }
  public static boolean parseArgs(String args[]) {
    if (args.length < 3)
      return false;
    OPT.dbPath_ = args[0];

    OPT.startURL_ = args[1];
    try {
      String executionMin = args[2];
      OPT.executionDurationMS_ = Integer.parseInt(executionMin) * 60 * 1000;
    } catch (NumberFormatException e) {
      System.err.println("The second argument, <execution_hour> should be a number ");
      return false;
    }
    return true;
  }
  public static void printUsage() {
    System.out.println("DocRobot <db_path> <starting_url> <execution_min>");
    System.out.println(" - Collects documents starting from starting_url ");
  }

  public class HtmlListener implements TextListener, URLListener, KeywordListener, TitleListener {
    private long srcUID_;
//  KeywordExtractor keyExt_ = new KoreanKeywordExtractor();
    KeywordExtractor keyExt_ = new SimpleKeywordExtractor();

    public HtmlListener() {
    }

    public void setPageUID(long srcUID) {
      srcUID_ = srcUID;
    }
    /*
     *  insert into keyword index
     *  @param uid - the URL identifier
     *  @param key - the keyword
     */
    public void onText(String text, int offset) throws PaposException {
      if ( text != null ) {
//      System.err.println("On Text("+text+") on UID("+srcUID_+")");

        if ( ! text.equals("") ) {
//        System.err.println("Inserting Text("+text+") on UID("+srcUID_+")");
          /* PAIS_BUG: Choose an appropriate keyword extractor by referring to the language of the text 
           */
          keyExt_.setText(text, offset);
          keyExt_.setKeywordListener(this);
          keyExt_.extractAll();
        }
      }
    }
    public void onTitle(String title) throws PaposException {
      if ( title != null ) {
        title = title.trim();
        if ( title.length() > 0 ) {
          if ( AdvRandomAccessFile.isStringWritable(title, HtmlPage.MAX_TITLE_LEN) ) {
//          System.err.println("Settting Title("+title+") on UID("+srcUID_+")");
            HtmlPageTable pageTable = HtmlPageTable.getInstance();
            HtmlPage page = pageTable.getPage(srcUID_);
            page.setTitle(title);
            pageTable.update(page);
          }
        }
      }
    }

    public void onKeyword(String keyword, int offset) throws PaposException {
      keyword = keyword.trim();
      if ( keyword.length() > 0 )
//      System.err.print(" <"+keyword+"> ");
        if ( AdvRandomAccessFile.isStringWritable(keyword, Keyword.MAX_KEYWORD_LEN) ) {
//        REMOVE			
//        System.err.println("Before Inserting KEYWORD("+keyword +" )" );
//        PaisSystem.printMemStat();

          // Store all keywords to lower case letters
          // Later, queries will be lower cased right before searching
          keyword = keyword.toLowerCase();
          KeywordTable.getInstance().insert(keyword,srcUID_,offset);

//        REMOVE			
//        System.err.println("After Inserting KEYWORD("+keyword +" )" );
//        PaisSystem.printMemStat();

        }
    }
    public void onURL(String URL) throws PaposException  {
//    System.err.println("On URL("+URL+") on UID("+srcUID_+")");
      if ( skipURL(URL) ) {
        return;
      }

      // getAbsoluteURL puts http://hostname:port prefix on the given URL
      // PAIS_BUG:remove filtering enas.com
      String absURL = httpConn_.getAbsoluteURL(URL).trim();
      /*
			String lowercaseURL = absURL.toLowerCase();
			if (  lowercaseURL.startsWith("http://www.enas.com") ||
				  lowercaseURL.startsWith("http://enas.com") ||
				  lowercaseURL.startsWith("http://cs95.enas.com") )
       */			
      if (absURL.length() > 0)
      {
        if ( AdvRandomAccessFile.isStringWritable(absURL, HtmlPage.MAX_URL_LEN) ) {
          insertPage( absURL );
        }
      }
    }
    private boolean skipURL(String URL) {
      if ( URL.length() > 10 ) {
        if ( URL.substring(0,11).compareToIgnoreCase("javascript:") ==0 ) {
          return true;
        }
      }
      return false;
    }
  }

  /* A object who listens to events occurred by parsing a HTML file*/ 
  private HtmlListener htmlListener_ = new HtmlListener();

  /* HTTP Connector ; connect to a web server to get HTML text of a URL */
  private HttpConnection httpConn_ = new HttpConnection();

  /* Insert a new page marking that it is not indexed yet. */
  public void insertPage(String URL) throws PaposException {
    KeywordTable  keywordTable = KeywordTable.getInstance();
    HtmlPageTable pageTable = HtmlPageTable.getInstance();
    HtmlPage page = pageTable.getPage( URL );

    // Insert Into HtmlPageTable if the page does not exist.
    if ( page == null ) {
      long UID = PAISSystemTable.getNextUID();
      /* HtmlPage.NULL_CHANGE_DATE means the document is not indexed yet */

      System.err.print("Trying to insert Link URL("+URL+",UID="+UID+")..." );

      pageTable.insert(UID, URL, HtmlPage.NULL_CHANGE_DATE);

      System.err.println("OK" );
    }
  }
  /*
   * @return HtmlPage of the URL if the document needs to be indexed
   *         null otherwise.
   */
  public HtmlPage getHtmlPage( String URL, long docChangeDate ) throws PaposException  {
    KeywordTable  keywordTable = KeywordTable.getInstance();
    HtmlPageTable pageTable = HtmlPageTable.getInstance();
    HtmlPage page = pageTable.getPage( URL );

    if ( page == null ) {
      long UID = PAISSystemTable.getNextUID();
      /* HtmlPage.NULL_CHANGE_DATE means the document is not indexed yet */
      /* PAIS_OPT: Change insert to return Object that is inserted */
      page = pageTable.insert(UID, URL, HtmlPage.NULL_CHANGE_DATE);

//    System.err.println("After inserting =>"+HtmlPageTable.getInstance().getStorage().getIndexDump(0));		

    }
    else {
      if ( page.isIndexed() ) {
        if ( page.getChangeDate() == docChangeDate ) {
          // already indexed with same page data.
          return null;
        } else {
          // already indexed but page data has been changed.
          keywordTable.deleteAll( page.getUID() );
          // indicate that HtmlPage is not indexed
          page.setChangeDate(HtmlPage.NULL_CHANGE_DATE);
          pageTable.update(page);
        }
      }
    }
    return page;
  }

  public void increaseConnectFailCount(String urlString) throws PaposException  {
    HtmlPageTable pageTable = HtmlPageTable.getInstance();
    HtmlPage unreachablePage = pageTable.getPage(urlString);
    if ( unreachablePage != null ) {
      unreachablePage.setConnectFailCount(unreachablePage.getConnectFailCount());
      pageTable.update(unreachablePage);
    }
  }
  public void removePageIfExist(String urlString) throws PaposException  {
    HtmlPageTable pageTable = HtmlPageTable.getInstance();
    HtmlPage invalidPage = pageTable.getPage(urlString);
    if ( invalidPage != null ) {
      pageTable.delete( invalidPage );
    }
  }
  /**
   * Index a HTML Page
   * @param page  The HTML Page to index
   * @return true if the page is successfully indexed
   *         false otherwise
   */
  public boolean indexHtmlPage(HtmlPage page) throws PaposException  {
    HtmlParser   parser = getHTMLParser(httpConn_, page.getURL() );
    if( parser == null ) {
      // invalid URL.
      // set Changed Date of the URL as INVALUD_URL_CHANGE_DATE
      // to skip the URL from indexing
      page.setChangeDate( HtmlPage.INVALUD_URL_CHANGE_DATE );
      HtmlPageTable.getInstance().update(page);
      return false;
    }

    if ( page.getChangeDate() != httpConn_.docChangeDate() ) { // The page needs to be indexed.
      indexKeywordsAndInsertLinks(page, parser);
    }

    httpConn_.disconnect();
    return true;
  }

  /**
   * Index a HTML Page 
   * @param urlString URL of HTML Page to index.
   */
  public boolean indexHtmlPage(String urlString) throws PaposException  {

    HtmlParser   parser = getHTMLParser(httpConn_, urlString);
    if ( parser == null) {
      // Invalid page. Do nothing
      return false;
    }
    /* Return a HtmlPage searched by urlString
     * Insert a HtmlPage into HtmlPageTable if not existent.
     */
    HtmlPage page = getHtmlPage( urlString, httpConn_.docChangeDate() );

    if ( page != null ) { // The page needs to be indexed
      indexKeywordsAndInsertLinks(page, parser );
    }
    httpConn_.disconnect();
    return true;
  }

  private HtmlParser htmlParser_ = null;
  /**
   * Connect to a HTML page URL to create a HTML parser on the document. 
   * 
   * @param conn      HTTP Connection object
   * @param urlString URL of the HTML Page
   * @return          HTML Parser that parses the content of HTML document.
   */
  private HtmlParser getHTMLParser(HttpConnection conn, String urlString) throws PaposException  {
    HtmlParser   parser = null;

    try {
      conn.connect(urlString);

      try {
        Reader reader = conn.getReader();

        if ( htmlParser_ == null ) {
          htmlParser_ = new HtmlParser( reader );
        } else {
          htmlParser_.ReInit(reader);
        }
        parser = htmlParser_;

      } catch ( IOException e) {
        System.err.println("IOException while reading html content.");
        e.printStackTrace();
        // Connection Fail => Increase Connection Fail Count
        increaseConnectFailCount(urlString);
        conn.disconnect();
      } catch ( Exception e) {
        // PAIS_BUG:remove the url.
        System.err.println("exception while reading html content.");
        e.printStackTrace();
        // Connection Fail => Increase Connection Fail Count
        increaseConnectFailCount(urlString);
        conn.disconnect();
      }
    } catch ( MalformedURLException e) {
      // Invalid URL => Remove URL from HtmlPageTable if exists.
      removePageIfExist(urlString);
    } catch ( IOException e) {
      // Connection Fail => Increase Connection Fail Count
      increaseConnectFailCount(urlString);
    }

    return parser;
  }
  /** 
   * Index a HTML Page
   * @param page   The HTML Page to index
   * @param parser The HTML Parser that is connected to the stream to the actual HTML content.
   */
  void indexKeywordsAndInsertLinks( HtmlPage page, HtmlParser parser) throws PaposException  {
    /* Set document URL ID */
    htmlListener_.setPageUID(page.getUID());
    parser.textListener_ = htmlListener_;
    parser.urlListener_ = htmlListener_;
    parser.titleListener_ = htmlListener_;

    long docChangeDate = httpConn_.docChangeDate();
    try {
      parser.html();
    } catch (ParseException e) {
      // PAIS_BUG : Log error.
      System.err.println("Error while parsing HTML document");
      e.printStackTrace();
      docChangeDate = HtmlPage.HTML_PARSER_ERROR_CHANGE_DATE;
    } catch (TokenMgrError e) {
      // PAIS_BUG : Log error.
      System.err.println("Error while tokenizing HTML document");
      e.printStackTrace();
      docChangeDate = HtmlPage.HTML_PARSER_ERROR_CHANGE_DATE;
    }

    /* Mark the page as an indexed page by setting change date of HTML document */
    /* Read page again, to update on the latest data */
    HtmlPageTable pageTable = HtmlPageTable.getInstance();
    page = pageTable.getPage( page.getUID() ); 
    page.setChangeDate( docChangeDate );

//  REMOVE			
//  System.err.println("Before Updating HTmlPage("+page.getUID()+" )" );
//  PaisSystem.printMemStat();

    pageTable.update(page);

//  REMOVE			
//  System.err.println("After Updating HTmlPage("+page.getUID()+" )" );
//  PaisSystem.printMemStat();

//  System.err.println("After updating date =>"+HtmlPageTable.getInstance().getStorage().getIndexDump(0));		

  }
}
